from naie.datasets import get_data_reference
import moxing as mox
import zipfile
import os

# 安装第三方库
os.system('python -m pip install --upgrade pip')
os.system('pip install torch==1.3.1')
os.system('pip install torchvision==0.4.2')
os.system('pip install pillow==6.2.1')
os.system('pip install tensorboardX tensorboard')
os.system('pip install pandas')

# 从线上数据湖中导入数据至本工程目录
data_ref = get_data_reference(dataset="DatasetService", dataset_entity='OceanDataset')  # 数据库名称和数据实例名称
for file_path in data_ref.get_files_paths():
    mox.file.copy(file_path, './raw/' + file_path.split('/')[-1])  # 拷贝到当前工程的raw文件夹下

# 解压图片zip压缩包
zip_file = zipfile.ZipFile('./raw/data.zip')
zip_list = zip_file.namelist()
for f in zip_list:
    zip_file.extract(f, './raw/')

# 从自己上传的压缩包中导入annotation.csv
data_ref = get_data_reference(dataset="DatasetService", dataset_entity='ocean_annotation')  # 数据库名称和数据实例名称
for file_path in data_ref.get_files_paths():
    mox.file.copy(file_path, './cache/' + file_path.split('/')[-1])  # 拷贝到当前工程的cache文件夹下

# 解压annotation的zip压缩包
zip_file = zipfile.ZipFile('./cache/ocean_annotation.zip')
zip_list = zip_file.namelist()
for f in zip_list:
    zip_file.extract(f, './cache/')

# 移至raw文件夹中
mox.file.copy('./cache/annotation.csv', './raw/annotation.csv')
